{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Language Translation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, random_split, DataLoader\n",
    "\n",
    "import torchtext\n",
    "from torchtext.vocab import Vocab\n",
    "from torchtext.data import Field, BucketIterator, TabularDataset\n",
    "\n",
    "from torchsummary import summary\n",
    "\n",
    "import spacy\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import os\n",
    "import time\n",
    "import math\n",
    "from PIL import Image\n",
    "import glob\n",
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.manual_seed(1)\n",
    "np.random.seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "HYPERPARAMETERS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 16\n",
    "LR = 3e-4\n",
    "NUM_EPOCHES = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Downloading trained pipelines from spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !python -m spacy download en_core_web_sm\n",
    "# !python -m spacy download de_core_news_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp_english = spacy.load(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp_german = spacy.load(\"de_core_news_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenizer_english(text):\n",
    "    return [token.text for token in nlp_english.tokenizer(text)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenizer_german(text):\n",
    "    return [token.text for token in nlp_german.tokenizer(text)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Hi', 'guys', ',', 'my', 'name', 'Jeff']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer_english(\"Hi guys, my name Jeff\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I', 'dont', 'know', 'any', 'German']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer_german(\"I dont know any German\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "ENGLISH = torchtext.data.Field(tokenize=tokenizer_english, lower=True, init_token=\"<sos>\", eos_token=\"<eos>\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "GERMAN = torchtext.data.Field(tokenize=tokenizer_german, lower=True, init_token=\"<sos>\", eos_token=\"<eos>\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "train, validation, test = torchtext.datasets.Multi30k.splits(exts=(\".de\", \".en\"), fields=(GERMAN, ENGLISH))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "ENGLISH.build_vocab(train, max_size=10000, min_freq=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "GERMAN.build_vocab(train, max_size=10000, min_freq=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ENGLISH vocab_size:  9797\n",
      "GERMAN vocab_size:  10004\n"
     ]
    }
   ],
   "source": [
    "print(\"ENGLISH vocab_size: \", len(ENGLISH.vocab))\n",
    "print(\"GERMAN vocab_size: \", len(GERMAN.vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataloader, validation_dataloader, test_dataloader = torchtext.data.BucketIterator.splits(\n",
    "    (train, validation, test),\n",
    "    batch_size=BATCH_SIZE,\n",
    "    sort_within_batch=True,\n",
    "    sort_key=lambda x: len(x.src),\n",
    "    device=device,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "torch.Size([13, 16])\n",
      "torch.Size([17, 16])\n"
     ]
    }
   ],
   "source": [
    "for batch_idx, data in enumerate(train_dataloader):\n",
    "    print(batch_idx)\n",
    "    print(data.src.size())\n",
    "    print(data.trg.size())\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def german2english(model, german_sentence, device=\"cpu\", max_len=100):\n",
    "    model.eval()\n",
    "    tokens = [token.text.lower() for token in nlp_german(german_sentence)]\n",
    "    tokens = [\"<sos>\"] + tokens + [\"<eos>\"]\n",
    "    \n",
    "    indexes = [GERMAN.vocab.stoi[token] for token in tokens]\n",
    "    indexes_tensor = torch.LongTensor(indexes).unsqueeze(1).to(device)\n",
    "    \n",
    "    english_sentence = [ENGLISH.vocab.stoi[\"<sos>\"]]\n",
    "    \n",
    "    for i in range(max_len):\n",
    "        trg = torch.LongTensor(english_sentence).unsqueeze(1).to(device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            word = model(indexes_tensor, trg)\n",
    "            \n",
    "        top = word.argmax(-1)[-1, :].item()\n",
    "        english_sentence.append(top)\n",
    "\n",
    "        if top == ENGLISH.vocab.stoi[\"<eos>\"]:\n",
    "            break\n",
    "\n",
    "    english_sentence = [ENGLISH.vocab.itos[word] for word in english_sentence]\n",
    "    \n",
    "    return english_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ENGLISH.vocab.stoi[\"<pad>\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformer_package.models import Transformer, Transformer_with_nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "source_vocab_size = len(GERMAN.vocab)\n",
    "target_vocab_size = len(ENGLISH.vocab)\n",
    "embed_size = 512\n",
    "num_head = 8\n",
    "num_ff = 1024\n",
    "encoder_layers = 3\n",
    "decoder_layers = 3\n",
    "hidden_size = 512\n",
    "dropout = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Transformer(\n",
       "  (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "  (encoder_embed): Embedding(10004, 512)\n",
       "  (decoder_embed): Embedding(9797, 512)\n",
       "  (encoder_positional_encoding): PositionalEncoding(\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (decoder_positional_encoding): PositionalEncoding(\n",
       "    (dropout): Dropout(p=0.1, inplace=False)\n",
       "  )\n",
       "  (encoders): ModuleList(\n",
       "    (0): Transformer_Encoder(\n",
       "      (Norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (Norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (multi_attention): MultiHeadAttention(\n",
       "        (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "        (Q): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (K): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (V): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (linear): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "      (feed_forward): Sequential(\n",
       "        (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (1): ReLU()\n",
       "        (2): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "      (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (1): Transformer_Encoder(\n",
       "      (Norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (Norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (multi_attention): MultiHeadAttention(\n",
       "        (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "        (Q): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (K): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (V): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (linear): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "      (feed_forward): Sequential(\n",
       "        (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (1): ReLU()\n",
       "        (2): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "      (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (2): Transformer_Encoder(\n",
       "      (Norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (Norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (multi_attention): MultiHeadAttention(\n",
       "        (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "        (Q): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (K): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (V): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (linear): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "      (feed_forward): Sequential(\n",
       "        (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (1): ReLU()\n",
       "        (2): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "      (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "  )\n",
       "  (decoders): ModuleList(\n",
       "    (0): Transformer_Decoder(\n",
       "      (masked_multiheadattention): MultiHeadAttention(\n",
       "        (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "        (Q): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (K): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (V): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (linear): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "      (multiheadattention): MultiHeadAttention(\n",
       "        (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "        (Q): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (K): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (V): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (linear): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "      (Norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (Norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (Norm3): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "      (feed_forward): Sequential(\n",
       "        (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (1): ReLU()\n",
       "        (2): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "    )\n",
       "    (1): Transformer_Decoder(\n",
       "      (masked_multiheadattention): MultiHeadAttention(\n",
       "        (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "        (Q): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (K): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (V): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (linear): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "      (multiheadattention): MultiHeadAttention(\n",
       "        (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "        (Q): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (K): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (V): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (linear): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "      (Norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (Norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (Norm3): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "      (feed_forward): Sequential(\n",
       "        (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (1): ReLU()\n",
       "        (2): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "    )\n",
       "    (2): Transformer_Decoder(\n",
       "      (masked_multiheadattention): MultiHeadAttention(\n",
       "        (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "        (Q): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (K): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (V): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (linear): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "      (multiheadattention): MultiHeadAttention(\n",
       "        (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "        (Q): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (K): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (V): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (linear): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "      (Norm1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (Norm2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (Norm3): LayerNorm((512,), eps=1e-05, elementwise_affine=True)\n",
       "      (dropout_layer): Dropout(p=0.1, inplace=False)\n",
       "      (feed_forward): Sequential(\n",
       "        (0): Linear(in_features=512, out_features=512, bias=True)\n",
       "        (1): ReLU()\n",
       "        (2): Linear(in_features=512, out_features=512, bias=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (final): Linear(in_features=512, out_features=9797, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = Transformer(source_vocab_size, target_vocab_size, embed_size, num_head, num_ff, encoder_layers, decoder_layers, hidden_size, dropout=dropout, device=device).to(device)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dimenstions of Input Source Vector:  torch.Size([100, 16])\n",
      "Dimenstions of Input Target Vector:  torch.Size([90, 16])\n",
      "Dimenstions of Predicted Vector:  torch.Size([90, 16, 9797])\n",
      "Dimenstions of Input Source Vector:  torch.Size([100, 16])\n",
      "Dimenstions of Input Target Vector:  torch.Size([100, 16])\n",
      "Dimenstions of Predicted Vector:  torch.Size([100, 16, 9797])\n",
      "Dimenstions of Input Source Vector:  torch.Size([100, 16])\n",
      "Dimenstions of Input Target Vector:  torch.Size([110, 16])\n",
      "Dimenstions of Predicted Vector:  torch.Size([110, 16, 9797])\n"
     ]
    }
   ],
   "source": [
    "def test(size):\n",
    "    sample_in_x = torch.rand(100, BATCH_SIZE).type(torch.LongTensor).to(device)\n",
    "    sample_in_y = torch.rand(size, BATCH_SIZE).type(torch.LongTensor).to(device)\n",
    "    sample_out = model(sample_in_x, sample_in_y)\n",
    "    print(\"Dimenstions of Input Source Vector: \", sample_in_x.size())\n",
    "    print(\"Dimenstions of Input Target Vector: \", sample_in_y.size())\n",
    "    print(\"Dimenstions of Predicted Vector: \", sample_out.size())\n",
    "    \n",
    "test(90)\n",
    "test(100)\n",
    "test(110)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(model.parameters(), lr=LR)\n",
    "criterion = nn.CrossEntropyLoss(ignore_index = ENGLISH.vocab.stoi[\"<pad>\"])\n",
    "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<sos>', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a']\n",
      "-------------------------------------------------\n",
      "Epoch: 1 Train mean loss: 0.30136652\n",
      "       1 Test  mean loss: 0.25347116\n",
      "-------------------------------------------------\n",
      "['<sos>', 'a', 'group', 'of', 'people', 'are', 'playing', 'in', 'a', 'group', 'of', 'people', 'are', 'playing', 'in', 'a', 'large', 'group', 'of', 'people', 'are', 'playing', 'in', 'a', 'large', 'group', 'of', 'people', 'are', 'playing', 'in', 'a', 'large', 'group', 'of', 'people', 'are', 'playing', 'in', 'a', 'large', 'group', 'of', 'people', 'are', 'playing', 'in', 'a', 'large', 'group', 'of', 'people', 'are', 'playing', 'in', 'a', 'large', 'group', 'of', 'people', 'are', 'playing', 'in', 'a', 'large', 'group', 'of', 'people', 'are', 'playing', 'in', 'a', 'large', 'group', 'of', 'people', 'are', 'playing', 'in', 'a', 'large', 'group', 'of', 'people', 'are', 'playing', 'in', 'a', 'large', 'group', 'of', 'people', 'are', 'playing', 'in', 'a', 'large', 'group', 'of', 'people', 'are']\n",
      "-------------------------------------------------\n",
      "Epoch: 2 Train mean loss: 0.27612641\n",
      "       2 Test  mean loss: 0.24889550\n",
      "-------------------------------------------------\n",
      "['<sos>', 'a', 'crowd', '.', '<eos>']\n",
      "-------------------------------------------------\n",
      "Epoch: 3 Train mean loss: 0.26799422\n",
      "       3 Test  mean loss: 0.24763601\n",
      "-------------------------------------------------\n",
      "['<sos>', 'a', 'street', '.', '<eos>']\n",
      "-------------------------------------------------\n",
      "Epoch: 4 Train mean loss: 0.26328001\n",
      "       4 Test  mean loss: 0.24974432\n",
      "-------------------------------------------------\n",
      "['<sos>', 'a', 'man', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man']\n",
      "-------------------------------------------------\n",
      "Epoch: 5 Train mean loss: 0.26003523\n",
      "       5 Test  mean loss: 0.23633513\n",
      "-------------------------------------------------\n",
      "['<sos>', 'a', 'large', 'crowd', '.', '<eos>']\n",
      "-------------------------------------------------\n",
      "Epoch: 6 Train mean loss: 0.25686562\n",
      "       6 Test  mean loss: 0.25257338\n",
      "-------------------------------------------------\n",
      "['<sos>', 'a', 'group', 'of', 'people', 'are', 'walking', 'in', 'a', 'group', 'of', 'people', 'are', 'walking', 'in', 'a', 'group', 'of', 'people', 'are', 'walking', 'in', 'a', 'group', 'of', 'people', 'are', 'walking', 'in', 'a', 'group', 'of', 'people', 'are', 'walking', 'in', 'a', 'group', 'of', 'people', 'are', 'walking', 'in', 'a', 'group', 'of', 'people', 'are', 'walking', 'in', 'a', 'group', 'of', 'people', 'are', 'walking', 'in', 'a', 'group', 'of', 'people', 'are', 'walking', 'in', 'a', 'group', 'of', 'people', 'are', 'walking', 'in', 'a', 'group', 'of', 'people', 'are', 'walking', 'in', 'a', 'group', 'of', 'people', 'are', 'walking', 'in', 'a', 'group', 'of', 'people', 'are', 'walking', 'in', 'a', 'group', 'of', 'people', 'are', 'walking', 'in', 'a', 'group']\n",
      "-------------------------------------------------\n",
      "Epoch: 7 Train mean loss: 0.25439377\n",
      "       7 Test  mean loss: 0.24766125\n",
      "-------------------------------------------------\n",
      "['<sos>', 'a', 'city', '.', '<eos>']\n",
      "-------------------------------------------------\n",
      "Epoch: 8 Train mean loss: 0.25264208\n",
      "       8 Test  mean loss: 0.25700124\n",
      "-------------------------------------------------\n",
      "['<sos>', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a', 'man', 'in', 'a']\n",
      "-------------------------------------------------\n",
      "Epoch: 9 Train mean loss: 0.25046009\n",
      "       9 Test  mean loss: 0.24009867\n",
      "-------------------------------------------------\n",
      "['<sos>', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a', 'man', 'is', 'a']\n",
      "-------------------------------------------------\n",
      "Epoch: 10 Train mean loss: 0.24913985\n",
      "       10 Test  mean loss: 0.23558781\n",
      "-------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "loss_hist = {}\n",
    "loss_hist[\"train loss\"] = []\n",
    "loss_hist[\"test loss\"] = []\n",
    "\n",
    "for epoch in range(1, NUM_EPOCHES+1):\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    epoch_train_loss = 0\n",
    "    epoch_test_loss = 0  \n",
    "    \n",
    "    for batch_idx, data in enumerate(train_dataloader):\n",
    "        x = data.src.to(device)\n",
    "        y = data.trg.to(device)\n",
    "        \n",
    "        y_pred = model(x, y[:-1, :])\n",
    "        \n",
    "        y_pred = y_pred.reshape(-1, y_pred.size(-1))\n",
    "        y = y[1:, :].reshape(-1)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss = criterion(y_pred, y)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        epoch_train_loss += loss.item()\n",
    "        \n",
    "    with torch.no_grad():\n",
    "        model.eval()\n",
    "        \n",
    "        for batch_idx, data in enumerate(validation_dataloader):\n",
    "            x = data.src.to(device)\n",
    "            y = data.trg.to(device)\n",
    "        \n",
    "            y_pred = model(x, y[:-1, :])\n",
    "            y_pred = y_pred.reshape(-1, y_pred.size(-1))\n",
    "            y = y[1:, :].reshape(-1)\n",
    "        \n",
    "            loss = criterion(y_pred, y)\n",
    "        \n",
    "            epoch_test_loss += loss.item()\n",
    "    \n",
    "    epoch_train_loss = epoch_train_loss / len(train_dataloader.dataset)\n",
    "    epoch_test_loss = epoch_test_loss / len(validation_dataloader.dataset)\n",
    "    \n",
    "    loss_hist[\"train loss\"].append(epoch_train_loss)\n",
    "    loss_hist[\"test loss\"].append(epoch_test_loss)\n",
    "    \n",
    "    if epoch%1 == 0:\n",
    "        print(german2english(model, \"Zwei junge weiße Männer sind im Freien in der Nähe vieler Büsche.\", device=device)) # Two young, White males are outside near many bushes.\n",
    "        print(\"-------------------------------------------------\")\n",
    "        print(\"Epoch: {} Train mean loss: {:.8f}\".format(epoch, epoch_train_loss))\n",
    "        print(\"       {} Test  mean loss: {:.8f}\".format(epoch, epoch_test_loss))\n",
    "        print(\"-------------------------------------------------\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAk5ElEQVR4nO3deXjV5Z338fc3GyGBJGRhyQmBIEEEDAlNAa1i0dpqtQZtp+pYO221VrvYTp+nU6fXTGfm6bN0ZjpT2xm7UGo7nWnttFaWVutSa0WLgAhhCTsEJAlLwpKEJWT7Pn+cAwY80ADn5JfkfF7XdS7Ob0u+OZfmk/t3/+77NndHRETkbElBFyAiIv2TAkJERKJSQIiISFQKCBERiUoBISIiUaUEXUAs5efn+/jx44MuQ0RkwHjjjTea3L0g2rFBFRDjx49n1apVQZchIjJgmNnucx3TLSYREYlKASEiIlEpIEREJCoFhIiIRKWAEBGRqBQQIiISlQJCRESiSviAaOvoYv7SHSzb3hR0KSIi/UrCB0RqchI/eKWW/3htV9CliIj0KwkfEMlJxgfKCnlpcyPNxzuCLkdEpN+Ia0CY2U1mtsXMtpvZI1GOV5nZOjOrNrNVZnZNb6+NparyQtq7unm2Zm88v42IyIASt4Aws2TgMeBmYApwt5lNOeu0F4Hp7l4OfAJYcAHXxkxZUTYl+ZksWtMQr28hIjLgxLMFMRPY7u473b0d+DlQ1fMEdz/qby2KnQl4b6+NJTPjtumFLK89yL7mtnh9GxGRASWeAREC9vTYrovsO4OZ3W5mm4GnCbcien1t5PoHIrenVjU2Nl50sfMqQrjDr9eqFSEiAvENCIuyz9+2w32hu08G5gFfu5BrI9fPd/dKd68sKIg6pXmvlORnMr0om0XV9Rf9NUREBpN4BkQdMLbHdhFwzj/P3X0pcJmZ5V/otbFyW3mImoYWth9ojfe3EhHp9+IZEK8DpWZWYmZpwF3Akp4nmNlEM7PI+xlAGnCwN9fGwwemjyHJYHG1bjOJiMQtINy9E/gs8BywCfiFu9eY2YNm9mDktA8CG8ysmvBTS3d6WNRr41XrKSOHp/Ouifksrm7grb5zEZHEZIPpF2FlZaVf6pKjv1y1hy89uY6nPn01M4pHxKgyEZH+yczecPfKaMcSfiT12W6aNpohKUksXqPOahFJbAqIswxPT+U9V4ziN+v20tnVHXQ5IiKBUUBEcVt5IQePtfOqZngVkQSmgIji3ZcXkJWeoqeZRCShKSCiGJKSzPuvHMNzNfs40d4VdDkiIoFQQJxDVXmI4+1dvLBpf9CliIgEQgFxDrNKchmdlc4STb0hIglKAXEOSUnGbeWF/GFLI4ePtQddjohIn1NAnEdVeSGd3c7T67WQkIgkHgXEeUwZk0XpyGEs0dNMIpKAFBDnYWZUlReyctch6g4fD7ocEZE+pYD4E6rKw+sU/XqtbjOJSGJRQPwJY3MzmFGcw2I9zSQiCUYB0QvzKkJs3tfK5n0tQZciItJnFBC9cMuVY0hOMk29ISIJRQHRC3nDhnBtaT5Lqhvo7h4862eIiJyPAqKX5pWHqD9yglW7DwddiohIn1BA9NKNU0YxNDVZndUikjAUEL2UOSSFG6eM4un1e2nv1EJCIjL4KSAuwLyKQo4c7+CVbY1BlyIiEncKiAtwbWkBIzJSWaSnmUQkASggLkBqchK3lI3hhY37OHqyM+hyRETiSgFxgeaVh2jr6OaFjfuCLkVEJK4UEBdoRvEIQjlDWbRGt5lEZHBTQFygpKTwDK+vbm+i6ejJoMsREYkbBcRFmFcRoqvbeXqdZngVkcFLAXERJo0azuTRw1mkQXMiMogpIC7SvIoQa948wpsHtZCQiAxOCoiLdNv0QgBNvSEig1ZcA8LMbjKzLWa23cweiXL8HjNbF3ktM7PpPY593sw2mFmNmX0hnnVejMKcocwsyWVRdT3umuFVRAafuAWEmSUDjwE3A1OAu81sylmn1QLXuXsZ8DVgfuTaacAngZnAdOBWMyuNV60Xa155iB2Nx6hp0EJCIjL4xLMFMRPY7u473b0d+DlQ1fMEd1/m7qfmz14OFEXeXwEsd/fj7t4JvAzcHsdaL8r7rxxNarLpNpOIDErxDIgQsKfHdl1k37ncB/w28n4DMMfM8swsA3g/MDbaRWb2gJmtMrNVjY19O4leTkYa100ayZK1DXRpISERGWTiGRAWZV/U36JmNpdwQHwZwN03Af8IvAA8C6wFok5+5O7z3b3S3SsLCgpiUfcFmVdRyP6Wk6yoPdjn31tEJJ7iGRB1nPlXfxHwtvkpzKwMWABUufvp37Lu/kN3n+Huc4BDwLY41nrRbpg8isy0ZBZr6g0RGWTiGRCvA6VmVmJmacBdwJKeJ5hZMfAUcK+7bz3r2Mge59wBPBHHWi/a0LRk3jdtNM9s2MvJzq6gyxERiZm4BUSkc/mzwHPAJuAX7l5jZg+a2YOR074K5AHfMbNqM1vV40v8ysw2Ar8GPtOjM7vfmVceorWtk5c2ayEhERk8UuL5xd39GeCZs/Z9r8f7+4H7z3HttfGsLZauviyP/GFpLK6u56Zpo4MuR0QkJjSSOgZSkpO4tayQFzcfoKWtI+hyRERiQgERI1XlhbR3dvPsBi0kJCKDgwIiRsrH5jAuL4MlWq9aRAYJBUSMmBlV5SGW7WjiQEtb0OWIiFwyBUQMVZUX0u2wZK1aESIy8CkgYuiygmFcGcpWQIjIoKCAiLGq8kLW1TWzs/Fo0KWIiFwSBUSMfWB6IWawSJ3VIjLAKSBibFRWOldflscSLSQkIgOcAiIOqqaH2HXwOGvrmoMuRUTkoikg4uCmK0eTlpKkhYREZEBTQMRBVnoqN0weya/X7qWzqzvockRELooCIk6qygtpOnqSZTu0kJCIDEwKiDh59+UjGZ6ewmI9zSQiA5QCIk7SU5N5/7QxPFezj7YOLSQkIgOPAiKOqsoLOXqykxc3HQi6FBGRC6aAiKNZE/IYlTWERXqaSUQGIAVEHCUnGR8oK+QPWw5w5Hh70OWIiFwQBUSczasI0dHl/FYLCYnIAKOAiLOphVlcVpDJojW6zSQiA4sCIs5OLSS0ovYQDUdOBF2OiEivKSD6QFV5IQC/1joRIjKAKCD6wLi8TCqKczQFuIgMKAqIPlI1vZBNe1vYur816FJERHpFAdFHbp1eSHKSaYZXERkwFBB9JH/YEK6ZmM/i6gYtJCQiA4ICog9VlRdSd/gEq988HHQpIiJ/kgKiD7136mjSU5NYtEad1SLS/ykg+tCwISm854pRPL1+Lx1aSEhE+rm4BoSZ3WRmW8xsu5k9EuX4PWa2LvJaZmbTexz7SzOrMbMNZvaEmaXHs9a+Mq88xKFj7by6rSnoUkREzituAWFmycBjwM3AFOBuM5ty1mm1wHXuXgZ8DZgfuTYEPAxUuvs0IBm4K1619qU5kwrIyUjVDK8i0u/FswUxE9ju7jvdvR34OVDV8wR3X+bup3pslwNFPQ6nAEPNLAXIAAbFjfu0lCTef+UYnq/Zz/H2zqDLERE5p3gGRAjY02O7LrLvXO4Dfgvg7vXAN4A3gb1As7s/H+0iM3vAzFaZ2arGxsaYFB5v88pDnOjo4oWN+4MuRUTknOIZEBZlX9QBAGY2l3BAfDmyPYJwa6MEKAQyzewj0a519/nuXunulQUFBTEpPN4qx40glDNUM7yKSL8Wz4CoA8b22C4iym0iMysDFgBV7n4wsvs9QK27N7p7B/AUcHUca+1TSUnGB6YXsnRbEwePngy6HBGRqOIZEK8DpWZWYmZphDuZl/Q8wcyKCf/yv9fdt/Y49CYw28wyzMyAG4BNcay1z82rKKSr23lm/d6gSxERiSpuAeHuncBngecI/3L/hbvXmNmDZvZg5LSvAnnAd8ys2sxWRa5dATwJrAbWR+qcH69agzB5dBaXjxrOYs3wKiL9lA2meYEqKyt91apVQZfRa9/5w3b+6dktvPJXcxmbmxF0OSKSgMzsDXevjHZMI6kDdNv08EJCS7SQkIj0QwqIABWNyOCd40ewaE29ZngVkX5HARGwqvIQ2w4cZdNeLSQkIv2LAiJgt1w5hhQtJCQi/VCvAsLMMs0sKfJ+kpndZmap8S0tMYzITOO6SQUsWdtAd7duM4lI/9HbFsRSID0yid6LwMeBH8erqERTVRFib3MbK3cdCroUEZHTehsQ5u7HgTuAf3P32wnP0Cox8J4rRpKRlqwxESLSr/Q6IMzsKuAe4OnIvpT4lJR4MtJSeN/U0Tyzfi/tnVpISET6h94GxBeAvwYWRkZDTwBeiltVCaiqvJDmEx0seHVn0KWIiAC9bAW4+8vAywCRzuomd384noUlmmtLC7hp6mj+6dktHDvZyf987+WEp6ESEQlGb59i+pmZZZlZJrAR2GJmX4pvaYklOcl47J4Z3D2zmMde2sEjv1pPp9atFpEA9fYW0xR3bwHmAc8AxcC98SoqUSUnGf/39mk8fP1E/nvVHh766WraOrqCLktEElRvAyI1Mu5hHrA4skaDHtqPAzPji++9nH+4bSq/27Sfj/5wJc0nOoIuS0QSUG8D4vvALiATWGpm44CWeBUl8BdXj+fbd1WwZs9h7vz+axxoaQu6JBFJML0KCHf/truH3P39HrYbmBvn2hLeB6YX8qOPzWTPoePc8d1l1DYdC7okEUkgve2kzjazfzWzVZHXvxBuTUicXVOazxMPzOZEexcf+u4y1tc1B12SiCSI3t5iehxoBT4cebUAP4pXUXKmsqIcfvngVaSnJnPX/Nd4dVtT0CWJSALobUBc5u5/5+47I69/ACbEszA504SCYTz16asZm5vBx3+8kt+s07QcIhJfvQ2IE2Z2zakNM3sXcCI+Jcm5jMpK578/dRUVY0fwuSfW8JPXdgVdkogMYr2dT+lB4Cdmlh3ZPgz8RXxKkvPJHprKT+6byeeeWMNXF9fQ1HqSv7xxkkZdi0jM9fYpprXuPh0oA8rcvQK4Pq6VyTmlpybz3XtmcGflWL79++18ZeEGurSWhIjE2AXNyBoZTX3KF4FHY1qN9FpKchJf/+CV5A9P47GXdnD4WDuP3lVOempy0KWJyCBxKUuO6p5GwMyML71vMn/3gSk8W7OPj/1oJS1tGnUtIrFxKQGhexr9xMffVcK37ipn1a7D3Pn95Rxo1ahrEbl05w0IM2s1s5Yor1agsI9qlF6oKg/x+Mfeye6Dx/jQd19j90GNuhaRS3PegHD34e6eFeU13N21olw/M2dSAT/75Gxa2zr44HeXsaFeo65F5OJdyi0m6YfKx+bw5ENXMyQlmbvmL2fZdo26FpGLo4AYhC4rGMavHrqawpx0Pvaj13lm/d6gSxKRAUgBMUiNzk7nl5+6mrKibD7zs9X81/LdQZckIgNMXAPCzG4ysy1mtt3MHoly/B4zWxd5LTOz6ZH9l5tZdY9Xi5l9IZ61DkbZGan8532zuP7ykfzNog08+rutuOvhMxHpnbgFhJklA48BNwNTgLvNbMpZp9UC17l7GfA1YD6Au29x93J3LwfeARwHFsar1sFsaFoy37/3HXzoHUU8+rtt/O1ijboWkd6J55NIM4Ht7r4TwMx+DlQBG0+d4O7Lepy/HCiK8nVuAHZEFimSi5CSnMQ/f6iM/GFD+N7LOzh0rJ1v3lnOkBSNuhaRc4vnLaYQsKfHdl1k37ncB/w2yv67gCfOdZGZPXBqIaPGxsaLKjQRmBmP3DyZv7nlCp5Zv4+P/+h1WjXqWkTOI54BEW0qjqj3NsxsLuGA+PJZ+9OA24BfnuubuPt8d69098qCgoJLKDcx3H/tBL5553RW1h7i7h8sp7H1ZNAliUg/Fc+AqAPG9tguAt62yo2ZlQELgCp3P3jW4ZuB1e6+P25VJqDbK4r4wV9UsuPAMT70vWW8efB40CWJSD8Uz4B4HSg1s5JIS+AuYEnPE8ysGHgKuNfdt0b5GndznttLcvHmXj6Sn35yFs0nOvjg95axsaHlT18kIgklbgHh7p3AZ4HngE3AL9y9xsweNLMHI6d9FcgDvhN5nHXVqevNLAO4kXCASBzMKB7Bkw9eRWqScef3X2P5zrMbcCKSyGwwPRdfWVnpq1at+tMnyhkajpzgo4+v5M1Dx/n2XeXcNG1M0CWJSB8xszfcvTLaMY2kFgpzhvLLT13F1MIsPv3T1fxsxZtBlyQi/YACQgAYkZnGT++fxXWTCvjKwvV8ZMEK1u45EnRZIhIgBYSclpGWwvyPVvI3t1zBxr0tVD32Rx76rzfYfqA16NJEJADqg5CoWts6+OGrtSx4pZbj7Z18cEYRX7hxEqGcoUGXJiIxdL4+CAWEnNehY+1856Xt/GT5bnC4Z3Yxn5k7kfxhQ4IuTURiQAEhl6zhyAm+9btt/PKNPaSnJnP/NSXcP2cCWempQZcmIpdAASExs6PxKP/6/FaeXr+XnIxUPv3uy/joVeNJT9XEfyIDkQJCYm59XTP//PwWlm5tZHRWOg/fUMqfVRaRmqznHkQGEo2DkJi7siibn3xiJj9/YDaFOel8ZeF63vvNpSxZ20C31psQGRQUEHJJZk/I41cPXc2Cj1YyJCWJh59Yw63/9iovbT6g1etEBjgFhFwyM+M9U0bx9MPX8uid5Rw92cnHf/w6H/7+a7y+61DQ5YnIRVJASMwkJxnzKkL87ovX8bWqqew6eJw/+95rfOLHr2u2WJEBSJ3UEjfH2zv5j2W7+e4fttPS1slt0wv54o2TGJ+fGXRpIhKhp5gkUM0nOpi/dAePv7qLjq5uPvzOsTx8fSmjs9ODLk0k4SkgpF840NrGv/9+O0+sfJMkMz529XgevO4yRmSmBV2aSMJSQEi/sufQcb75wlYWVtczLC2FB+ZM4BPXlJA5JCXo0kQSjgJC+qUt+1r5xvNbeGHjfvKHpfGZuRP581nFDEnRqGyRvqKAkH5t9ZuH+ednt/DazoOEcobyhfeUcseMIpKTLOjSRAY9jaSWfm1G8Qh+9slZ/Od9M8nNTONLT67jfY8u5TfrGmjv7A66PJGEpZu+0i+YGdeWFnDNxHye3bCPbzy/hc/+bA0jMlK5tayQeRUhZhTnYKZWhUhf0S0m6Zc6u7p5ZXsTC1fX8/zGfbR1dDM+L4N5FSFurwgxLk9jKURiQX0QMqC1tnXw7IZ9LFxTz2s7D+IO7xg3gtsrQtxaNoacDD0mK3KxFBAyaDQcOcHi6gYWrqlj6/6jpCUnMXdyAbdXFDF3coGegBK5QAoIGXTcnZqGFhauqWdxdQNNR0+SPTSVW8vGcMeMEDOKR6i/QqQXFBAyqHV2dfPq9iYWrqnnuZpwf8W4vAzmlYf7KzT3k8i5KSAkYRw92Rnpr6hj2Y5wf0VFcQ53VIS4taxQ03qInEUBIQlpb3Okv2J1PVv2t5KabLz78pHcURHi+itGqr9CBAWEJDh3Z+PeFhaurmfx2gYaW0+SlZ7CLWWF3DEjROU49VdI4lJAiER0dTt/jPRXPLthHyc6uhibO5Tby0PcPqOIEvVXSIIJLCDM7CbgW0AysMDdv37W8XuAL0c2jwIPufvayLEcYAEwDXDgE+7+2vm+nwJCLsSxk508VxMeX/HH7U10O5SPzeGOGeH+ilz1V0gCCCQgzCwZ2ArcCNQBrwN3u/vGHudcDWxy98NmdjPw9+4+K3LsP4BX3H2BmaUBGe5+5HzfUwEhF2t/SxuLq+t5anU9m/e1kpIU6a+YEeL6ySNJT1V/hQxOQQXEVYR/4b8vsv3XAO7+/85x/ghgg7uHzCwLWAtM8AsoUAEhsbBpb3h8xaI19RxoPcnw9BRunjaaeeUhZk3I0yyzMqicLyDiOVlfCNjTY7sOmHWe8+8Dfht5PwFoBH5kZtOBN4DPu/uxsy8ysweABwCKi4tjULYkuivGZHHFmCy+fNNklu1oYtGaBp5Zv49frKpjVNYQbpteSFV5iKmFWerclkEtni2IPwPe5+73R7bvBWa6++einDsX+A5wjbsfNLNKYDnwLndfYWbfAlrc/W/P9z3VgpB4aevo4neb9rNoTQMvbz1AR5czceQwqiJhUZyXEXSJIhclqBZEHTC2x3YR0HD2SWZWRrgz+mZ3P9jj2jp3XxHZfhJ4JI61ipxXemoyt5YVcmtZIYePtfPMhr0srm7gX17Yyr+8sJUZxTnMqwhxy5VjyBs2JOhyRWIini2IFMKd1DcA9YQ7qf/c3Wt6nFMM/B74qLsvO+v6V4D73X2Lmf09kOnuXzrf91QLQvpa/ZETLKluYHF1uHM7OcmYU5pPVXmI904dRUaallyR/i3Ix1zfDzxK+DHXx939/5jZgwDu/j0zWwB8ENgduaTzVKFmVk64ZZEG7AQ+7u6Hz/f9FBASpM37Wli0poEl1fU0NLcxNDWZ904dxbzyENeU5pOarAUcpf/RQDmRPtTd7azafZhF1fU8s34vR453kJuZxq1lY6gqL9RMs9KvKCBEAtLe2c3LWxtZXF3PCxv3c7Kzm7G5Q6maHmJeRSETRw4PukRJcAoIkX7g6MlOntuwj0XVb43cnjImi3kVhdw2PcTo7PSgS5QEpIAQ6WcOtLbxm7V7Wby2gbV7jmAGs0vymFdRyE3TxpA9NDXoEiVBKCBE+rHapmMsrg6vjFfbdOz0MqrzykPM1TQfEmcKCJEBwN1ZV9fM4uoGfr0uPC25pvmQeFNAiAwwnV3dvLbzIIvWNPBczT6OnuwkJyOVaybmM6e0gDmTCtRnITGhgBAZwNo6unhx0wF+v/kAr2xr5EDrSQAmjRp2OixmluTqVpRcFAWEyCDh7mzZ38rSrY0s3drEyl2HaO/sZkhKEjNLcrluUjgwSkcO01gL6RUFhMggdaK9ixW1B1m6tYml2xrZfuAoAGOy07m2NJ9rSwu4ZmI+I7T4kZyDAkIkQdQfOcErWxtZuq2RV7c10dLWiRmUFeVwXWk+cyYVUD42hxRN+yERCgiRBNTZ1c26+ubI7ahGqvccodth+JAUrp6Yx5xJBcwpLWBsrqYqT2QKCBGh+XgHf9zRxCvbwv0X9UdOADAhP5M5kwq4tjSf2RPyyByiGWgTiQJCRM7g7uxoPBZuXWxrZPnOg7R1dJOabFSOyw23Liblc8XoLJI09mJQU0CIyHm1dXTxxu7DLN3ayMtbG9m8rxWA/GFDuLY0nzmTwh3e+VoMadBRQIjIBTnQ0sbSbeHbUa9sa+LQsXYAphZm8Y5xI5hWmM3UUBaTRg3XOhcDnAJCRC5ad7dT09DC0m2NvLKtkQ31LRw92QlAWnISk8cMZ2phNtNCWUwrzOby0cM1aG8AUUCISMx0dzu7Dx1nfX0zNfXNbGhoZkN9C80nOgBISTJKRw1nWmEW00Lh4LhiTJaWX+2nFBAiElfuTt3hE2zoERgb6ps5GLk1lWRwWcEwpoWymRoJjimFWWSla1rzoJ0vIBTpInLJzIyxuRmMzc3g5ivHAOHQ2N9ykvX1zWyob6amoZnXdhxk4Zr609eV5GeeDoxpheHw0Kjv/kMBISJxYWaMzk5ndHY6N04ZdXp/Y+tJNjREbk/Vt1C95wi/Wbf39PGiEUOZFunTmBoJjoLhenoqCAoIEelTBcOHMPfykcy9fOTpfUeOt4dvSzWcam208GzNvtPHR2UN4cpQdqQzPJsrQ9ma7rwPKCBEJHA5GWlcU5rPNaX5p/e1tnVQ09ByOjA21Dfz+80H6I50m4ZyhjJrQi6zSnKZVZLHuLwMzWAbYwoIEemXhqenMntCHrMn5J3ed7y9k017W1m75wiv7zrEy1saeWp1uE9jVNYQZpbkMbMkl9kluUzUlOeXTE8xiciAFZ4y5CjLdx5iZe0hVtQeZH9LeEGl3Mw0Zo7PZdaEXGaW5DJ5dJaWbI1CTzGJyKBkZkwcOZyJI4fzkdnjcHfePHScFTsPsSISGKf6MrLSU3jn6cDIY1phlqY9/xMUECIyaJgZ4/IyGZeXyYffORYIr5GxsvZguIWx8xAvbj4AQEZaMu8YN4LZE8K3pcqKshmSohHgPekWk4gklAOtbaysPXQ6MLbsD09MOCQliYriHGaV5DGrJJeK4hEMTRv8gaGR1CIi53D4WDsrd73Vh7GxoYVuh9Rko6woh1kl4T6MyvG5DBuEa2UoIEREeqmlrYM3dh1meeS21Pq6Zjq7nSSDaaHsSGDkMXN8LtkZA3+qkMACwsxuAr4FJAML3P3rZx2/B/hyZPMo8JC7r40c2wW0Al1A57l+gJ4UECISa8fbO1m9+wgrag+yovYQ1XuO0N7ZjRmUjhzGxJHDKMnPpCR/GCX5GZTkD2NERuqAecQ2kKeYzCwZeAy4EagDXjezJe6+scdptcB17n7YzG4G5gOzehyf6+5N8apRRORPyUhLOWMQX1tHF2v3HGFlJCw2723l+Zr9dHa/9cd29tBUSvIzmZCfSUl+JuMj/5bkZw6oJV3jWelMYLu77wQws58DVcDpgHD3ZT3OXw4UxbEeEZFLlp6azKwJeczqMYCvo6ub+sMnqG06xs6mY9Q2HaW26RjLdx7kqR6TE0J4QN/ZLY6S/EyKczNIS+lfj93GMyBCwJ4e23Wc2To4233Ab3tsO/C8mTnwfXefH+0iM3sAeACguLj4kgoWEbkYqclJjI+0FOaedexEexe7Dx2jtvFUeIRfz9XsO71SH4SnRB+bm8H4vHBLY0LBW62OwuyhgawNHs+AiPbTRO3wMLO5hAPimh673+XuDWY2EnjBzDa7+9K3fcFwcMyHcB/EpZctIhI7Q9OSmTw6i8mjs952rPl4B7UHIy2OHgGyatchjrV3nT4vLSWJkrxMxkdaHBPyMymJBEheZlrc+jviGRB1wNge20VAw9knmVkZsAC42d0Pntrv7g2Rfw+Y2ULCt6zeFhAiIgNVdkYq5Rk5lI/NOWO/u9PYevKMFsfOxmNsP3CU328+QEfXW38LDx+SwuQxw/nFp66KeVDEMyBeB0rNrASoB+4C/rznCWZWDDwF3OvuW3vszwSS3L018v69wP+KY60iIv2GmTEyK52RWelnTFYI0NnVTcORNnZG+jlqm45FnqqKfSsibgHh7p1m9lngOcKPuT7u7jVm9mDk+PeArwJ5wHciP9ypx1lHAQsj+1KAn7n7s/GqVURkoEhJTqI4L4PivAzefXl8v5cGyomIJLDzjYPoX89UiYhIv6GAEBGRqBQQIiISlQJCRESiUkCIiEhUCggREYlKASEiIlENqnEQZtYI7L7Iy/MBTS0eps/iTPo8zqTP4y2D4bMY5+4F0Q4MqoC4FGa2qjeLEiUCfRZn0udxJn0ebxnsn4VuMYmISFQKCBERiUoB8ZaoCxIlKH0WZ9LncSZ9Hm8Z1J+F+iBERCQqtSBERCQqBYSIiESV8AFhZjeZ2RYz225mjwRdT5DMbKyZvWRmm8ysxsw+H3RNQTOzZDNbY2a/CbqWoJlZjpk9aWabI/+NXBV0TUEys7+M/H+ywcyeMLP0oGuKtYQOCDNLBh4DbgamAHeb2ZRgqwpUJ/A/3P0KYDbwmQT/PAA+D2wKuoh+4lvAs+4+GZhOAn8uZhYCHgYq3X0a4VUz7wq2qthL6IAAZgLb3X2nu7cDPweqAq4pMO6+191XR963Ev4FEAq2quCYWRFwC7Ag6FqCZmZZwBzghwDu3u7uRwItKngpwFAzSwEygIaA64m5RA+IELCnx3YdCfwLsSczGw9UACsCLiVIjwJ/BXQHXEd/MAFoBH4UueW2wMwygy4qKO5eD3wDeBPYCzS7+/PBVhV7iR4QFmVfwj/3a2bDgF8BX3D3lqDrCYKZ3QoccPc3gq6ln0gBZgDfdfcK4BiQsH12ZjaC8N2GEqAQyDSzjwRbVewlekDUAWN7bBcxCJuJF8LMUgmHw0/d/amg6wnQu4DbzGwX4VuP15vZfwVbUqDqgDp3P9WifJJwYCSq9wC17t7o7h3AU8DVAdcUc4keEK8DpWZWYmZphDuZlgRcU2DMzAjfY97k7v8adD1Bcve/dvcidx9P+L+L37v7oPsLsbfcfR+wx8wuj+y6AdgYYElBexOYbWYZkf9vbmAQdtqnBF1AkNy908w+CzxH+CmEx929JuCygvQu4F5gvZlVR/Z9xd2fCa4k6Uc+B/w08sfUTuDjAdcTGHdfYWZPAqsJP/23hkE47Yam2hARkagS/RaTiIicgwJCRESiUkCIiEhUCggREYlKASEiIlEpIEQugJl1mVl1j1fMRhOb2Xgz2xCrrydyqRJ6HITIRTjh7uVBFyHSF9SCEIkBM9tlZv9oZisjr4mR/ePM7EUzWxf5tziyf5SZLTSztZHXqWkaks3sB5F1Bp43s6GB/VCS8BQQIhdm6Fm3mO7scazF3WcC/054Jlgi73/i7mXAT4FvR/Z/G3jZ3acTntPo1Aj+UuAxd58KHAE+GNefRuQ8NJJa5AKY2VF3HxZl/y7genffGZnwcJ+755lZEzDG3Tsi+/e6e76ZNQJF7n6yx9cYD7zg7qWR7S8Dqe7+v/vgRxN5G7UgRGLHz/H+XOdEc7LH+y7UTygBUkCIxM6dPf59LfJ+GW8tRXkP8Grk/YvAQ3B63eusvipSpLf014nIhRnaY6ZbCK/RfOpR1yFmtoLwH153R/Y9DDxuZl8ivCLbqRlQPw/MN7P7CLcUHiK8MplIv6E+CJEYiPRBVLp7U9C1iMSKbjGJiEhUakGIiEhUakGIiEhUCggREYlKASEiIlEpIEREJCoFhIiIRPX/AfZIRD6ZmTD5AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss_hist[\"train loss\"])\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAA1mklEQVR4nO3deXxcdb34/9d7JnuaZLK1TZqk63RvlrasJYWCKCBQN6SA1YdfEVFQ3AHv1as/770qAnoRkMWrV1QoCCggRVaBIlubrftG2yxN2yTtZGmTNNvn98dM6rRNm0wyZ84s7+fjMY/MnDk58555NH3P+bzP5/0RYwxKKaXUSDnsDkAppVRk0cShlFIqIJo4lFJKBUQTh1JKqYBo4lBKKRWQOLsDCIWcnBwzZcoUu8NQSqmIUlFR0WKMyT1xe0wkjilTprBu3Tq7w1BKqYgiIrVDbdehKqWUUgHRxKGUUiogmjiUUkoFRBOHUkqpgGjiUEopFRBNHEoppQKiiUMppVRANHEopaLGm9ub2bq/3e4wop4mDqVUVGjv7uWGP6zjv1dvtTuUqKeJQykVFZ6tbqS7d4Ca+lYGBnSBOitp4lBKRYVVa+sQgbauXnYfPGJ3OFFNE4dSKuJt3NvGxr3tXHtmEQDVda32BhTlNHEopSLeqrV1JMY5+PaHZzEuMY6qeo/dIUU1TRxKqYjW1dPPM1WNXLYgj8zUBIoLMqiub7U7rKimiUMpFdFWb9hHx9E+rj6jEICyIhdb9nXQ1dNvc2TRSxOHUiqirVpbx9ScVM6amgVAaWEm/QOGjY1tNkcWvTRxKKUi1s6mw6zd4+HqMwoREQBKC10AVNVpncMqmjiUUhHr8bV1xDmETy4sOLYtNy2RgsxkrXNYSBOHUioi9fQN8FTlXj40ZwK5aYnHPVdWlEmVXpJrGU0cSqmI9PLmAxw60sPVZxae9FxpoYt9bd3sb+u2IbLop4lDKRWRVq2tIz8jiaXu3JOeKytyAVCt8zksoYlDKRVx6g918tbOFq5aXIjTISc9PzcvnXinUKV1Dkto4lBKRZw/r6sH4KrFBUM+nxTvZG5+htY5LKKJQykVUfoHDE+sa2CpO5eCzJRT7ldW6GJDQxt9/QMhjC42WJo4ROQSEdkmIjtF5LYhnr9ORNb7bm+LSInfc3tEZIOIVIvIOr/tPxSRvb7t1SJymZXvQSkVXt7Y3sT+9m5WnHFyUdxfWZGLrt5+th3oCFFksSPOqgOLiBO4D7gYaADWisizxpjNfrvtBs43xnhE5FLgIeAsv+eXGWNahjj8L4wxd1oVu1IqfK16v57s1AQumjPhtPsNTgSsrm9lXn5GCCKLHVaecZwJ7DTG7DLG9ACrgOX+Oxhj3jbGDF728C4w9IClUkoBTR3dvLq1iU8tKiAh7vT/fRVlpZCVmqB1DgtYmTgmAfV+jxt8207lC8ALfo8N8JKIVIjIDSfse7NveOu3IpI51MFE5AYRWSci65qbm0cTv1IqzDxZ0UD/gOHTwwxTAYgIpYUunUFuASsTx8nXyHmTwck7iizDmzhu9du8xBizELgUuElElvq2/xqYDpQC+4C7hjqmMeYhY8xiY8zi3NyTr/NWSkUWYwyPr63nzKlZTM8dN6LfKSt0sbPpMG1dvRZHF1usTBwNgP/XggKg8cSdRKQY+A2w3BhzcHC7MabR97MJ+AveoS+MMQeMMf3GmAHg4cHtSqno9s6ug9Qe7By2KO6v1DcRcH1DqzVBxSgrE8dawC0iU0UkAVgBPOu/g4gUAU8DK40x2/22p4pI2uB94MPARt/jPL9DfHxwu1Iquj2+tp60pDgunZ83/M4+JYUuRHQp2WCz7KoqY0yfiNwMvAg4gd8aYzaJyI2+5x8AfgBkA/f7WiL3GWMWAxOAv/i2xQGPGmP+7jv0HSJSinfYaw/wJaveg1IqPLR29vDCxv2sOKOQ5ATniH8vPSme6bnjdAZ5kFmWOACMMauB1Sdse8Dv/vXA9UP83i6g5MTtvudWBjlMpVSY+0vVXnr6Bo6t8heIskIXr25twhhzbM0ONTY6c1wpFdaMMax6v57igoxRzccoLXJx6EgPdYc6LYguNmniUEqFter6VrYd6BjV2QZAWWHmseOo4NDEoZQKa4+vrSc53smVJfmj+v2ZE8aRHO/UiYBBpIlDKRW2Dh/t49maRi4vziMtKX5Ux4hzOiguyNACeRBp4lBKha2/1TTS2dPPiiFW+QtEaZGLzY1tdPf2Bymy2KaJQykVtlatrcc9fhwLi4bsLDRiZYWZ9PYbNu9rD1JksU0TxzA6urVVgVJ22Lq/ner6Vq4+o3DMl9EOLiWrdY7g0MRxGj9ZvYWrHniHzp4+u0NRKuaser+eBKeDTywce9PsCelJ5Gck6ZVVQaKJ4zSWzMhh+4EOvvPkeowZsj+jUsoC3b39/KVqLx+eN4Gs1ISgHLO0yEVVnWf4HdWwNHGcxtKZuXz3ktk8v34fD7yxy+5wlIoZL27aT1tXLyvOKAraMcsKM2nwdNHccTRox4xVmjiG8aWl07i8OI87XtzKG9t1XQ+lQmHV+/UUZiVz7vTsoB1zsFOuDleNnSaOYYgId3yqmFkT0vjqo5XUHjxid0hKRbU9LUd4Z9dBrl5ciMMRvN5S8/MziHMI1fU6XDVWmjhGICUhjodWLkZEuOGRCo4c1WK5UlZ5fF09DoGrFo9t7saJkhOczM5L0yurgkATxwgVZadw77Vl7Gjq4DtP1mixXCkL9PYP8GRFAxfOHs+E9KSgH7+sMJP1DW30D+jf71ho4ghAuTuXWy+ZzeoN+/n1Gx/YHY5SUee1rU00dxzl6iAWxf2VFro4fLSPnU2HLTl+rNDEEaAblk7jipJ8fv7iNl7f1mR3OCrGbdvfwa9e3cFAlHyDfnxtPePTElk2K9eS45cdK5BrnWMsNHEESET42ScXMGtCGl97rIo9LVosV/b5v7f3cNfL27n/9Z12hzJm+9q6eH1bE1ctLiDOac1/TVNzUslIjtc6xxhp4hiFlIQ4Hv7sYhwO4YY/rNNiubJNZa0HEbj75e28t+ug3eGMyZ/XNTBg4OrF1gxTgfeLX0mhSy/JHSNNHKNUmJXCvdcsZGfTYS2WK1u0d/eyvamDG8qnMTk7la+tquLg4cic3DYwYHh8bT1LZmRTlJ1i6WuVFbrYdqCDw/qFb9Q0cYzBee4cbrvUWyy//3UtlqvQqq5rxRjvRRv3XluGp7OXbz5RE5H1jrd2trC3tcuyori/0iIXxsD6hlbLXytaaeIYoy+WT+PKknzufGkb/9BiuQqhiloPDoGSQu9a3N+/fC5vbG/moTWR1x7n8bX1uFLi+ci8CZa/VmmBC9BOuWOhiWOMvMXyYmZPTOcWLZarEKqs8zBrYvqxlfE+c1YRH12Qx89f3EZF7SGboxu5g4eP8tLm/XyirIDEOKflr5eZmsDUnFStc4yBJo4gSE5w8tDKRceK5Tp2qqzWP2Coqmtl0WTXsW0iwk8+uYBJrmS++mgVniM99gUYgKcr99Lbb8a8yl8gygpdVNW1am1ylDRxBMlxxfI/a7FcWWtHk7e4e+LKeOlJ8dx37UJaDvdExEUbxhgeW1vHwiIXMyekhex1S4tctBw+yt7WrpC9ZjTRxBFE57lzuP3SObywUYvlyloVtd4JbIsmn7yk6oKCDL532Wxe2dLE/761O9ShBWRdrYddzUeC2j59JMoKvZ+b1jlGRxNHkF1fPpXlpb5i+VYtlitrVNR6yBmXQFHW0Jeufu7cKXxk3gR++sLWsF68aNX79YxLjOOjxXkhfd3ZeWkkxjm0zjFKmjiCTET46SeKmTMxna+tqmK3FsuVBSprPSwsyjzlWtwiwh2fLGFiRhI3P1pFW2dviCMcXltXL89vaOSKknxSE+NC+trxTgcLJmWEdVINZ5o4LJCc4OTBlYuIcwg3PKLFchVcLYePsudg55DDVP4yUuL51TVlHGjvDst6x7M1jXT3DnBNCIvi/koLXWxsbKenb8CW149kmjgsUpiVwr3XLuSD5sN8+4nw+6NVkWtwXH7hMIkDoKwok9sunc1Lmw/wf2/vsTawAK16v445eeksmJRhy+uXFWXS0zfAln3ttrx+JNPEYaElM7zF8r9v2s99/4j8JnQqPFTUeoh3yoj/w/3CeVP50Jzx/PfqLWEzW3rj3jY2Nbaz4ozCUw63WU2Xkh09TRwWGyyW3/Xydl7besDucFQUqKz1MC8/g6T4kU2WExHuvKqE3HGJ3PxoFe3d9tc7Vq2tIzHOwcdKJ9kWQ35GEuPTErXOMQqaOCzmXyy/ZVW1FsvVmPT0DVDT0DpsfeNErpQEfnVtGXtbu7jtqfW2Dp129vTxTFUjly3IIyMl3rY4RIRS7ZQ7Kpo4QkCL5SpYtuxr52jfwEkT/0Zi0eQsvvORWazesJ8/vldnQXQjs3rDfjqO9rHiDHuK4v7KijLZc7CTQxEyyz5caOIIEf9i+beeqI7IDqbKfoMT/xb6tRoJxA3l07hgVi4//ttmNjW2BTGykVv1fh1Tc1I5c2qWLa/vr7TQBUCNnnUERBNHCC2ZkcP3LpvDi5sOaLFcjUpFnYdJrmTyMpJH9fsOh3D3p0vJSkng5kerQn72u7Opg3W1Hq62sSjur7ggA4dAlSaOgGjiCLEvnDeVj5Xmc/crWixXgaus9YzoMtzTyUpN4J5ryqg71Mn3nt4Q0nrH42vriXMIn1xYELLXPJ3UxDhmTkjTAnmANHGEmIjwk8Fi+WPV7Go+bHdIKkI0tnaxr62bhb7LSMfizKlZfPPimTxb08iqtfVjD24Ejvb181TlXj40ZwK5aYkhec2RKCvKpKa+VYePA2Bp4hCRS0Rkm4jsFJHbhnj+OhFZ77u9LSIlfs/tEZENIlItIuv8tmeJyMsissP3c2xfv2xwrFjuFG74QwUdYXB5pAp/lXWnbmw4Gl8+fzrl7hx++OymkEyCe2VzE4eO9IS0ffpIlBW6aO/uY5de8ThiliUOEXEC9wGXAnOBa0Rk7gm77QbON8YUAz8GHjrh+WXGmFJjzGK/bbcBrxpj3MCrvscRpzArhfuuXcjuliN8K0KX+1ShVVHrISnewZy89KAcz+EQfnF1KenJ8dz0aCVHLK53rFpbR35GEuXuXEtfJ1BlOhEwYFaecZwJ7DTG7DLG9ACrgOX+Oxhj3jbGDA4uvguMZOBzOfB73/3fAx8LTrihd+6MHG73tYO4V4vlahiVtR5KClzEO4P3Z5szLpH/WVHKnpYjfP+vGy2rd9Qf6mTNjhauWlyI02F/Udzf9NxxpCXGaZ0jAFYmjkmA/+Bpg2/bqXwBeMHvsQFeEpEKEbnBb/sEY8w+AN/P8UMdTERuEJF1IrKuubl5VG8gFAaL5b94ZTuvbtFiuRpad28/mxrbx1wYH8q503P42kVunq7ay58rGoJ+fIA/r6tHBD4dBnM3TuRwCCU6ETAgViaOob5WDPl1RkSW4U0ct/ptXmKMWYh3qOsmEVkayIsbYx4yxiw2xizOzQ2vU2N/g8XyuXnpfH1VNR9osfy02jp7+cM7e8KibUYorW9oo2/AsGgUE/9G4qsXujl3ejY/eGYj2w90BPXYff0DPLGugaXuXCa5RncZsdVKC11s3d9BV0+/3aFEBCsTRwPg//WiAGg8cScRKQZ+Ayw3xhwc3G6MafT9bAL+gnfoC+CAiOT5fjcPiPjVkgaL5fFxDm54ZJ0Wy4fQ1z/AH97ZwwV3/oPvP7OJP7xTa3dIIfWviX/WJA6nQ/jlilLGJcZx058q6ewJXr3jzR3N7G/vtq19+kiUFbnoHzBs2GvPpMhIY2XiWAu4RWSqiCQAK4Bn/XcQkSLgaWClMWa73/ZUEUkbvA98GNjoe/pZ4HO++58DnrHwPYRMQWYK915bxp6DnXxTi+XH+efOFj56z1t8/5lNzJ6YzpTsFNbsCN/hRytU1HqYlpNKVmqCZa8xPi2JX15dxs7mw/zHM5uCdtzH3q8nZ1wCF86eELRjBtvgDHKtc4yMZYnDGNMH3Ay8CGwBnjDGbBKRG0XkRt9uPwCygftPuOx2AvCWiNQA7wPPG2P+7nvup8DFIrIDuNj3OCqcO907s/xlLZYDUHvwCDc8so7rfvMenb19PPCZhTz6xbP4yPyJVNR6LL8KKFwYY6iq81Bm0TCVv/PcOdy8bAZ/rmjg6cqx1zua2rt5bWsTn1xYQEJc+E4byx6XSFFWitY5RsjS9RqNMauB1Sdse8Dv/vXA9UP83i6g5MTtvucOAhcFN9Lw8f+WTGHj3jZ+8cp25uWnc9Gc8P2WZpWO7l7u/cdOfvfWHuKdwncvmcX/WzL1WBvxpe5cHnxjF+/tPhjW32KDpfZgJweP9ARt/sZwbrnIzXu7D/Hvf91IcYGLGePHjfpYT1Y20D9guDoMi+InKi108f7uQ3aHERHC9ytAjPIWyxcwLz/2iuUDA4Yn1taz7M43ePCNXVxZms8/vn0BX7lgxnFrTyyanElSvIM3t7fYGG3oDNY3QpU44pwO7llRRlK8k5sfraS7d3QF44EBw+Nr6zlzahbTckeffEKlrMjF/vZu9rV12R1K2NPEEYaS4p08uHJxTBXL1+45xJX3vcV3n1rP5OwUnrlpCXdeVcL49KST9k2Kd3LW1OyYqXNU1HlIS4zDPYZv/oGamJHE3Z8uYev+Dn703OZRHePd3QepPdgZ1kVxf4N1jmrf0rzq1DRxhKlJrmTuu3Zh1BfL97Z2cfOjlVz1wDscPNzD/6wo5ckbz6HE90d8KuXuHD5oPsLe1uj/dlhZ66G0yIUjxBPnLpg1ni9fMJ3H3q/j2ZqTLogc1qr360lPiuPS+XkWRBd8c/PTSXA6tM4xApo4wtg507P5N1+x/Nt/rqGmvtXWlduCqbOnj7tf2saFd77OK1sOcMtFbl771gUsL500onbbg20r3orys46O7l62HegI2TDVib518UwWT87k9qfWB7R6pedID3/fuJ+Pl00a8RK3dkuMczI3P50qPeMYliaOMPf5JVO4/rypPLe+keX3/ZPzf/46P39xK1v3t0dkEjHG8NeqvVx45xvc89pOPjJvIq996wK+cfFMkhNG/h/MzAnjGJ+WyJs7orvOUV3fijGhq2+cKM7p4J5ryoiPc3DTn0Ze7/hL1V56+ge4+owiiyMMrrIiF+v3ttLXP2B3KGFNE0eYExH+/fK5rPu3i7njU8VMzk7hgTd2cckv1/DhX7zJPa/uiJh1zKvrW/nEr9/m649Xk5uWyJM3nsM915SRP4rZxCJCuTuXf+5soT9Kh/HAWxgX+df4ux3yXcnc/ekSNu9r57+e3zLs/sZ4i+LFBRnMzQ9OQ8ZQKS100d07wNb9wZ09H20svRxXBU9GSjyfXlzIpxcX0nL4KC9s2MdzNfu4++Xt3P3yduZPSueK4nwuL8kPu7YOB9q7+dnft/J05V5y0xK541PFfGphwZjH7JfOzOGpygY2NbZRXOAKTrBhprKulVkT0khLirc1jgtnT+CL5VN5eM1uzpmezWULTl23qK5vZduBDv774wtCGGFwDK7lXl3fyvxJGTZHE740cUSgnHGJrDxnCivPmcK+ti6eX7+P52oa+ckLW/nJC1tZNDmTK4rzuKw4j/FpJ1+VFCrdvf38Zs0u7n/9A/r6DV++YDo3LZvBuMTg/LNbMiMHgDU7WqIycQwMGKpqPVxRmm93KAB895LZrN3j4dYn1zMvP53J2alD7rfq/XqS451cURIZRXF/BZnJZKcmUFXXymfOnmx3OGFLh6oiXF5GMteXT+OZm8/jje9cwHc+MosjR/v44XObOfu/X+W637zLY+/X4TnSE7KYjDGs3rCPi+56gztf2k65O4dXvnk+t14yO2hJA7wJdF5+Om9uj84C+Y6mw3Qc7bOssWGg4p0OfnVNGSJw86NVHO07ud5x+Ggfz61v5IqSPNvPkkZDRCgtdFFdr61HTkcTRxSZnJ3KTctm8PevL+Wlbyzl5mUzaGzt5vanN3DGf73C53/3Pk9XNlg6L2RTYxtXP/QuX/lTJWlJcTx6/Vk8uHIxRdkplrxeuTuXyjoPh6Ow/UioJ/6NRGFWCj+/qoQNe9v4yeqtJz3/XE0jnT39EVcU91dW5OKD5iO0dUb//KnR0qGqKDVzQhrf/PAsvnHxTDY1tvNcTSPP1TTyj201JMQ5uHDWeK4szWfZrPEBXc10Ki2Hj3LXS9tYtbYeV3I8//mx+aw4o5C4IC46NJSl7hweeOMD3tt1MOras1TWechKTWCyRUl3tD4ybyKfXzKF3/1zD+dMz+Yj8yYee27V2nrc48cFZV10u5QWehN1TUMrS2eG75IMdtLEEeVEhPmTMpg/KYNbL5lNVb2H52r28bf1+/j7pv2kJji5eO4ErijJp9ydG3Ajup6+AX7/9h7ueXUHXb39fP7cqdxykZuMlNAMUyya4m0/smZHS/QljloPC4syRzSvJdRuv3QOFbUevvPnGubmpVOYlcKWfe3U1Lfy/cvnhmXMI1VcmIEIVNVp4jgVTRwxxOEQFk3OYtHkLL5/+Vze23WQ59Y3snrDfv5a3Xhslu8VJfmcPS3rtGcLxhhe3dLEf63ewu6WIyyblcu/fXTumBrijUZinJOzp2XzZpRNBDx0pIddLUe4anF4tutIiHNw7zUL+eg9a/jqY1U88aVzeHxtPQlOBx8vO91Cn+EvPSmeGbnjtM5xGpo4YpTTIZw7I4dzZ+Twoyvn88+dLTxX08jzG/bx+Drv+gmXLfAmkUVFmcddOrv9QAc//ttm1uxoYVpuKr/7/BksmzXkCr4hUe7O5cd/20yDp5OCzPAa1hmtyjCsb5yoKDuFOz5VzJf/VMl/Pr+ZZ6ob+cj8iZauGRIqZUUuXt58AGNMRJ89WUUThyIhzsGy2eNZNns83b39vL6tiedq9vH42noeeaeWvIwkLi/O4+K5E3l+fSN/fK+O1AQnP7h8LivPmUy8xXWM4Sx1ey/LfWtHCyvOjNyirL/KOg9xDqG4ILznEly6II/PnjOZR3wrMq6IgPbpI1FamMkT6xqoPdjJlJyhLzuOZZo41HGS4p1cMj+PS+bncfhoH69uOcBzNY3839t7eHjNbhwC155VxDcvnhU23yxnjB/HxPQk1kRR4qio9TAvPz0i+jx977I5VNW10tXbzznTsu0OJyjKfMX9qnqPJo4haOJQpzQuMY7lpZNYXjqJts5e3tjRzKwJacyamGZ3aMfxth/J4aXNB+gfMDhD3EU22Hr7B6hpaOWaCEmCSfFOnvryuXT39Ye8g69VZk5IIyXBSXVdKx8vK7A7nLCj8zjUiGSkxHNlSX7YJY1B5TNzaevqZcPeNrtDGbMt+9rp7h0I6/rGiRLiHKRH4IS/U3H6hgmrtMX6kDRxqKhw3owcRGBNFMwiHyyMLwyTGeOxqrQwk82N7aNeATGaaeJQUSErNYF5+emsiYI26xV1reRlJI2qa7AKnrIiF30Dhk2NkX8WG2wjShwikioiDt/9mSJypYhEz3mpigrR0n6kstbDwggapopWZb5W9rqw08lGesbxJpAkIpOAV4HPA/9nVVBKjUa5O4e+AcO7Hxy0O5RR29/Wzd7WrrBpbBjLxqcnMcmVrHWOIYw0cYgxphP4BPArY8zHgbnWhaVU4BZNziQ53smaCJ5FXlkX/hP/YklpoYtqPeM4yYgTh4icA1wHPO/bppfyqrDibT+SFdF1jopaD4lxDubkRdbKedGqrMjF3tYumjq67Q4lrIw0cXwduB34izFmk4hMA/5hWVRKjVK5O5ddLUeoP9RpdyijUlHroaTAFXCzSWWNwSV79azjeCP612mMecMYc6Ux5me+InmLMeZrFsemVMCWzvS1H9kZeWcd3b39bGps08J4GJk/KYM4h2id4wQjvarqURFJF5FUYDOwTUS+Y21oSgVueu448jKSIrLOsWFvG739RusbYSQp3smcvHQ94zjBSM+H5xpj2oGPAauBImClVUEpNVqD7Ufe2tFC/4CxO5yADE78K4vgRZCiUVmRi/UNrRH378lKI00c8b55Gx8DnjHG9AL6KaqwVO7Opb27j/UNrXaHEpCKWg9TslPIGZdodyjKT2mhiyM9/exo6rA7lLAx0sTxILAHSAXeFJHJQLtVQSk1FksG249E0NVVxhgq63TiXzgq882p0eGqfxlpcfweY8wkY8xlxqsWWGZxbEqNSlZqAgsmZURUnaPuUCcth3u0vhGGpmSn4EqJ1xnkfkZaHM8QkbtFZJ3vdhfesw+lwlK5O4fKulY6unvtDmVEBif+aWPD8CMi3omAemXVMSMdqvot0AF82ndrB35nVVBKjVW5O5f+AcM7EdJ+pKLWw7jEOGZOCM+29bGutNDF9qaOiPkiYrWRJo7pxpj/MMbs8t1+BEyzMjClxmJhUSYpCc6IqXNU1LZSVuSK+EWoolVZUSbGwIYG7ZQLI08cXSJy3uADEVkCdFkTklJjlxDn4Jxp2RFR5zh8tI9t+9t1mCqMlRa4AHQioM9I+03dCDwiIhm+xx7gc9aEpFRwnOfO4dWtTdQf6qQwK8XucE6ppr6VAYNeURXGMlLimZabqgVyn5FeVVVjjCkBioFiY0wZcKGlkSk1RuXuXCD8L8utqPUg8q++SCo8eQvkHozRKWwBdVIzxrT7ZpADfHO4/UXkEhHZJiI7ReS2IZ6/TkTW+25vi0jJCc87RaRKRP7mt+2HIrJXRKp9t8sCeQ8qdkzPTSU/AtqPVNR6mDk+jYxkXRstnJUVZdJyuIcGj47Sj6UF52mreCLiBO4DLsW7dsc1InLiGh67gfONMcXAj4GHTnj+FmDLEIf/hTGm1HdbParoVdTzth/J5Z87W+jrH7A7nCENDOjEv0hxbEVArXOMKXEMd752JrDTdxVWD7AKWH7cAYx52xjj8T18FygYfE5ECoCPAr8ZQ4wqxpXPzPG2H9kbnlfDfNB8mI7uPhZqf6qwN2tiGknxDp1BzjCJQ0Q6RKR9iFsHkD/MsScB9X6PG3zbTuULwAt+j38JfBcY6qvizb7hrd+KyJBf1UTkhsEJi83N4T1UoayzZLqv/cj28KxzVNTqin+RIt7pYMGkDKrqPcPvHOVOmziMMWnGmPQhbmnGmOGuyBpqKGvIsxQRWYY3cdzqe3w50GSMqRhi918D04FSYB9w1ylif8gYs9gYszg3N3eYUFW0ykxNoDiM249U1HrITIlnao42YogEZUWZbGps52hfv92h2MrKZcYagEK/xwVA44k7iUgx3uGo5caYwWm+S4ArRWQP3iGuC0XkjwDGmAPGmH5jzADwMN4hMaVOqdydS1V9K+1hOOu3os7DosmZiOjEv0hQWuiip2+ALftiu1OulYljLeAWkakikgCsAJ7130FEioCngZXGmO2D240xtxtjCowxU3y/95ox5jO+38nzO8THgY0WvgcVBcrdOWHZfsRzpIddzUeOdV9V4W9wrZTqutgerrIscRhj+oCbgRfxXhn1hG+98htF5Ebfbj8AsoH7fZfWrhvBoe8QkQ0ish5vh95vWBG/ih5lRZmkJjjDbrhqcKxc6xuRIy8jmQnpiTF/ZdVIZ46Piu9S2dUnbHvA7/71wPXDHON14HW/x7ryoApIQpyDc6Znh91EwIpaD06HUOJrZ6Eig3bKtXaoSqmwUe7OpfZgJ7UHj9gdyjEVtR7m5aeTnOC0OxQVgLKiTGoPdnLw8FG7Q7GNJg4VE8rdOUD4tB/p6x+gpr5NGxtGoMHWMDURtjRxMGniUDFhak4qk1zJYVPn2Lq/g67efp0xHoGKCzJwCDHd8FATh4oJIsLSmTm8vfNgWLQf0Yl/kSslIY5ZE9Njus6hiUPFjHJ3Lh1H+8JiiKGi1sPE9CTyM5LsDkWNQlmRi+q6VgYGYrNTriYOFTPOnZ6NQ8KjzuFtbOjSiX8RqrTQRcfRPna1HLY7FFto4lAxw5WSwIICl+2J40B7Nw2eLi2MR7DBppSVMVrn0MShYspSdw7V9a20ddnXfqRS6xsRb1rOONKS4mK2zqGJQ8WUcneu7e1HKmo9JMQ5mJefMfzOKiw5HEJpoStmr6zSxKFiSlmRy/b2I5V1HoonZZAQp39+kay00MW2/e109vTZHUrI6b9cFVPinQ7OmZ5jW52ju7efjXvbdZgqCpQVuRgwsL4hPBcJs5ImDhVzls7Moe6QPe1HNjW20dM/oBP/osBgj7FYrHNo4lAxp9ztXdjrTRvOOgYn/ukVVZEve1wik7NTqIrBFuuaOFTMmZKdQkFmMmu2h77OUVnbyuTsFHLTEkP+2ir4BgvkxsTWREBNHCrmiAjl7lze+eAgvSFsP2KMoaLOo2cbUaSs0EVTx1H2tXXbHUpIaeJQMWmpO8fbfiSE49MNni6aO45qfSOKlPq+BMRanUMTh4pJ507PwSGhrXMca2yoZxxRY25eOglxjpirc2jiUDEpIyWekkJXSOdzVNZ5SE1wMmtiWsheU1nLO5Ez9jrlauJQMavcnUtNfSttnaFpP1JR66G0yIXToY0No0lZYSbrG9pCWi+zmyYOFbOWunMYMPD2B9YPVx052seWfe06TBWFSotcHO0bYNv+DrtDCRlNHCpmlRS6SEuMC0mdo6a+lQGDFsajUJlvKdmqGBqu0sShYpa3/Ug2a3Y0W34dfqWveFqmZxxRpyAzmZxxCTFVINfEoWJa+cxcGjxd1B7stPR1Kmo9uMePIyM53tLXUaEnIpQWZsZUgVwTh4pp5TNyACy9umpgwFBZ16qNDaNYWZGLXc1HQnahhd00caiYNjk7hcKsZEvrHLtaDtPW1av1jSg2WOeoDoP17ENBE4eKaaFoP1JZ2wroin/RbEFBBiLETJ1DE4eKeUvdORw+2mfZGHVFrQdXSjzTclItOb6yX1pSPDPHp8VMnUMTh4p55/jaj1jVLXewsaGITvyLZqWFLqrrY6NTriYOFfMykuMpLXRZUudo7exhZ9NhHaaKAWVFLlo7e9lj8RV64UATh1J424+sb2iltbMnqMetqmsFdOGmWFBa5AJio86hiUMpvMvJetuPHAzqcSvrPDgdQklhRlCPq8KPe3waqQnOmKhzaOJQCu/60WmJcUGfz1FR62FOXhopCXFBPa4KP06HUFzgOnaWGc00cSgFxDkdnDsjmze3twStuNnXP0B1fas2NowhZUUutuxrp7u33+5QLKWJQymfcncue1u72N1yJCjH27q/g86efp34F0NKC130DRg27m2zOxRLaeJQymepOxeANUG6umqwSKqF8dgxWCCP9jqHJg6lfIqyU5icnRK0OkdFrYfxaYkUZCYH5Xgq/I1PS2KSKznq6xyaOJTyU+7OCVr7kYo6D4sm68S/WFNa5NIzjrEQkUtEZJuI7BSR24Z4/joRWe+7vS0iJSc87xSRKhH5m9+2LBF5WUR2+H7qOIAKmnJ3Lkd6+sf8jbGpo5v6Q1068S8GlRW62NvaRVN7t92hWMayxCEiTuA+4FJgLnCNiMw9YbfdwPnGmGLgx8BDJzx/C7DlhG23Aa8aY9zAq77HSgXFOdOzcTpkzMNVg40NdeGm2FPmq3NURvFwlZVnHGcCO40xu4wxPcAqYLn/DsaYt40xg9Ms3wUKBp8TkQLgo8BvTjjucuD3vvu/Bz4W/NBVrEpPiqcsCO1HKus8JDgdzJ+UHqTIVKSYl5+BKyWeX76ynSNH++wOxxJWJo5JQL3f4wbftlP5AvCC3+NfAt8FThxsnmCM2Qfg+zl+qIOJyA0isk5E1jU3W7dIj4o+57lzxtx+pKLWw4KCDBLjnEGMTEWCpHgn/7OijO0HOvj2n2uisumhlYljqIrgkJ+giCzDmzhu9T2+HGgyxlSM9sWNMQ8ZYxYbYxbn5uaO9jAqBpW7czEG/rlzdO1Hjvb1s6GhTesbMez8mbncfukcXti4n3tf22l3OEFnZeJoAAr9HhcAjSfuJCLFeIejlhtjBv9SlwBXisgevENcF4rIH33PHRCRPN/v5gFN1oSvYlVJQQZpSaNvP7KpsZ2e/gEW+sa6VWy6vnwqHy+bxF0vb+elTfvtDieorEwcawG3iEwVkQRgBfCs/w4iUgQ8Daw0xmwf3G6Mud0YU2CMmeL7vdeMMZ/xPf0s8Dnf/c8Bz1j4HlQMinM6WDI9hzU7Rtd+pLJWJ/4p7+qSP/nEAooLMvjG49VsP9Bhd0hBY1niMMb0ATcDL+K9MuoJY8wmEblRRG707fYDIBu4X0SqRWTdCA79U+BiEdkBXOx7rFRQlc/MYW9rF7tG0X6kotZDYVYy49OTLIhMRZKkeCcPrlxEckIcX3xkXdDb9tvF0nkcxpjVxpiZxpjpxpj/8m17wBjzgO/+9caYTGNMqe+2eIhjvG6Mudzv8UFjzEXGGLfv5yEr34OKTcfajwS4KqAxhopajzY2VMfkZSTz4MqFNLZ28dXHquizaG37UNKZ40oNoTArhSnZKQH3rdrb2kVTx1FtbKiOs2hyFv/5sfms2dHCz/6+1e5wxkwTh1KnUO7O5Z1dB+npG/k3xAqtb6hTuPqMIj53zmQeXrObpysb7A5nTDRxKHUK5e4cOnv6qQxgKdDKWg8pCU5mT0yzMDIVqf798rmcPS2L257eQE0E97PSxKHUKYym/UhFnYfSQhdxTv3TUieLdzq4/7pF5I5L5Et/qKCpIzL7Wem/bqVOIS0pnoVFrhHXOTp7+tiyr0OHqdRpZaUm8NBnF9HW1cuX/1jJ0b7IWy1QE4dSp1HuzmXD3jY8R4a/jLKmvo3+AaMzxtWw5uVncOdVJVTUevjBXzdFXFsSTRxKnUa5O8fbfuSD4c86BmshZTpjXI3AR4vzuHnZDB5fV88f3q21O5yAaOJQ6jSKC1ykJ8WxZvvwiaOi1sOM8eNwpSSEIDIVDb558Uw+NGc8P3puM+98MLreaHbQxKHUaTgdwnnuHNbsaD7tcIIxhso6nfinAuNwCL+4upSpOal85U8V1B/qtDukEdHEodQwyt25NLZ180HzqduP7Go5QmtnLwsnu0IXmIoKaUnxPPzZxfQPGL74yDo6e8J/DQ9NHEoN47wZOQCnvSx3cOKfFsbVaEzNSeVX1y6MmDU8NHEoNYzCrBSm5qSe9rLcyloPGcnxTMsZF8LIVDQ5f2Yut106m9Ub9nPfP8J7DQ9NHEqNQLk7h3c+OHjKa+4r6zwsLHLhcAy1fplSI/PF8ml8rDSfu17eziubD9gdzilp4lBqBMrduXT19lNZ23rSc21dvWw/cFgn/qkxExF++sli5udn8PXHq9nZFJ5reGjiUGoEzp6WRdwp2o9U1Wl9QwXP4BoeSfEOvvhIBW2dvXaHdBJNHEqNgLf9SOaQdY7KWg8OgZJCV+gDU1Ep35XMrz+ziAZPJ19dVUX/QHgVyzVxKDVC5e4cNja2cfDw0eO2V9a1MicvndTEOJsiU9HojClZ/H/L5/Pm9mbuCLM1PDRxKDVC5TNzfe1H/jXDt3/AUFXn0fqGssQ1Zxax8uzJPPjmLv5atdfucI7RxKHUCC2YlEFGcvxxy8lu29/BkZ5+rW8oy/zgirmcOTWLW59az/qGVrvDATRxKDViTodw3owc1uxoOTZBq0IL48pi8U4Hv75uITlhtIaHJg6lAlDuzmF/ezc7mw4DUFXrITctkYLMZJsjU9Ese1wiD312EZ7OHr7yx8qAljO2giYOpQJwnnuw/Yj36qoK38Q/EZ34p6w1uIbHuloP//HsRlvbkmjiUCoABZkpTMtNZc2OZpo7jlJ7sFOHqVTIXF6cz03LpvPY+/X88b062+LQxKFUgJa6c3l31yHe2+29ukoThwqlb108iwtnj+dHz27i3V32rOGhiUOpAJW7c+jq7efhN3eR4HQwLz/D7pBUDHE4hF+uKKUoO4Wv/KmSBk/o1/DQxKFUgM6elk28U6hpaGPepHSS4p12h6RiTLpvDY/e/gFueKQi5Gt4aOJQKkCpiXHHJvzpin/KLtNzx3HPNWVs2d/Od55cH9JiuSYOpUah3Hd1ldY3lJ2WzRrPrZfM5vn1+7j/9Q9C9rraXEepUfjEwgK2HTh87PJcpezypaXT2NzYzp0vbWP2xDQumjPB8tfUMw6lRiHflcyvrikjLSne7lBUjBMRfvbJYublp3PLqupjk1OtpIlDKaUiXHKCkwdXLiYp3sENj6yjrcvaNTw0cSilVBSY5FvDo97TyS0Wr+GhiUMppaLEGVOy+OGV83h9WzM/f3GbZa+jxXGllIoi1501mc2N7TzwxgfMyUtjeemkoL+GnnEopVSU+Y8r5nHmlCy+++R6NjS0Bf34mjiUUirKJMQ5uP8zCzlzahbJCcHvbKBDVUopFYVyxiXyhy+cZcmxLT3jEJFLRGSbiOwUkduGeP46EVnvu70tIiW+7Uki8r6I1IjIJhH5kd/v/FBE9opIte92mZXvQSml1PEsO+MQESdwH3Ax0ACsFZFnjTGb/XbbDZxvjPGIyKXAQ8BZwFHgQmPMYRGJB94SkReMMe/6fu8Xxpg7rYpdKaXUqVl5xnEmsNMYs8sY0wOsApb772CMedsY4/E9fBco8G03xpjB6Y/xvpt9y10ppZQ6xsrEMQmo93vc4Nt2Kl8AXhh8ICJOEakGmoCXjTHv+e17s29467ciMmSXORG5QUTWici65ubmUb8JpZRSx7MycQy1CPOQZw0isgxv4rj12I7G9BtjSvGehZwpIvN9T/0amA6UAvuAu4Y6pjHmIWPMYmPM4tzc3NG+B6WUUiewMnE0AIV+jwuAxhN3EpFi4DfAcmPMSesgGmNagdeBS3yPD/iSygDwMN4hMaWUUiFiZeJYC7hFZKqIJAArgGf9dxCRIuBpYKUxZrvf9lwRcfnuJwMfArb6Huf5HeLjwEYL34NSSqkTWHZVlTGmT0RuBl4EnMBvjTGbRORG3/MPAD8AsoH7RQSgzxizGMgDfu+7MssBPGGM+Zvv0HeISCneYa89wJeseg9KKaVOJqFcbtAuItIM1I7y13OAliCGE+n08/gX/SyOp5/H8aLh85hsjDmpSBwTiWMsRGSd7yxIoZ+HP/0sjqefx/Gi+fPQXlVKKaUCoolDKaVUQDRxDO8huwMIM/p5/It+FsfTz+N4Uft5aI1DKaVUQPSMQymlVEA0cSillAqIJo7TGG49kVghIoUi8g8R2eJbH+UWu2MKB75GnFUi8rfh945uIuISkSdFZKvv38k5dsdkFxH5hu/vZKOIPCYiSXbHFGyaOE7Bbz2RS4G5wDUiMtfeqGzTB3zLGDMHOBu4KYY/C3+3AFvsDiJM/A/wd2PMbKCEGP1cRGQS8DVgsTFmPt6uGSvsjSr4NHGc2rDricQKY8w+Y0yl734H3v8UTtciP+qJSAHwUbwNOmOaiKQDS4H/BTDG9Piak8aqOCBZROKAFIZo7hrpNHGcWqDricQEEZkClAHvDbNrtPsl8F1gwOY4wsE0oBn4nW/o7jcikmp3UHYwxuwF7gTq8C770GaMecneqIJPE8epjXg9kVghIuOAp4CvG2Pa7Y7HLiJyOdBkjKmwO5YwEQcsBH5tjCkDjgAxWRP0LSy3HJgK5AOpIvIZe6MKPk0cpzai9URihW/t96eAPxljnrY7HpstAa4UkT14hzAvFJE/2huSrRqABr9VOp/Em0hi0YeA3caYZmNML95lI861Oaag08RxasOuJxIrxNvz/n+BLcaYu+2Ox27GmNuNMQXGmCl4/128ZoyJum+VI2WM2Q/Ui8gs36aLgM02hmSnOuBsEUnx/d1cRBReKGDZehyR7lTridgcll2WACuBDb514AG+Z4xZbV9IKsx8FfiT70vWLuDzNsdjC2PMeyLyJFCJ92rEKqKw9Yi2HFFKKRUQHapSSikVEE0cSimlAqKJQymlVEA0cSillAqIJg6llFIB0cShVBCISL+IVPvdgjZzWkSmiMjGYB1PqbHSeRxKBUeXMabU7iCUCgU941DKQiKyR0R+JiLv+24zfNsni8irIrLe97PIt32CiPxFRGp8t8F2FU4Redi3zsNLIpJs25tSMU8Th1LBkXzCUNXVfs+1G2POBO7F21UX3/1HjDHFwJ+Ae3zb7wHeMMaU4O33NNitwA3cZ4yZB7QCn7T03Sh1GjpzXKkgEJHDxphxQ2zfA1xojNnlaxS53xiTLSItQJ4xpte3fZ8xJkdEmoECY8xRv2NMAV42xrh9j28F4o0x/xmCt6bUSfSMQynrmVPcP9U+Qznqd78frU8qG2niUMp6V/v9fMd3/23+taTodcBbvvuvAl+GY2uap4cqSKVGSr+1KBUcyX6dg8G7/vbgJbmJIvIe3i9q1/i2fQ34rYh8B+/qeYPdZG8BHhKRL+A9s/gy3pXklAobWuNQykK+GsdiY0yL3bEoFSw6VKWUUiogesahlFIqIHrGoZRSKiCaOJRSSgVEE4dSSqmAaOJQSikVEE0cSimlAvL/AxF+68D9J1p+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss_hist[\"test loss\"])\n",
    "plt.xlabel(\"Epoch\")\n",
    "plt.ylabel(\"Loss\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<sos>',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a',\n",
       " 'man',\n",
       " 'in',\n",
       " 'a']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "german2english(model, \"Ein Mann mit einem orangefarbenen Hut, der etwas anstarrt.\", device=device) # A man in an orange hat starring at something."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#torch.save(model, \"trained_models/language_translation_1.pt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}